import scrapy


class LianjiaSpider(scrapy.Spider):
    name = 'lianjia'
    allowed_domains = ['lianjia.com']
    start_urls = ['https://www.lianjia.com/city/']

    def parse(self, response):
        """爬取所有城市名，城市链接"""
        li_list = response.xpath('//ul[@class="city_list_ul"]/li/div[2]/div/ul/li')
        for li in li_list:
            item = dict()
            city_name = li.xpath('./a/text()').extract_first()  # 城市名
            city_url = li.xpath('./a/@href').extract_first()    # 城市链接
            item['city'] = city_name if city_url else ''
            item['url'] = city_url + "ershoufang/" if city_url else ''  # 补充完整城市链接
            item['name'] = 'city'
            yield item
            yield scrapy.Request(item['url'], callback=self.crawl_room)
            # print(item)

    def crawl_room(self, response):
        """爬取二手房的标题，地区，基本信息，关注人数和发布时间"""
        div_list = response.xpath('//li[@class="clear LOGVIEWDATA LOGCLICKDATA"]/div[1]')
        for div in div_list:
            item = dict()
            title = div.xpath('./div[1]/a/text()').extract_first()                  # 二手房标题
            url = div.xpath('./div[1]/a/@href').extract_first()                     # 二手房链接
            region = div.xpath('./div[2]/div/a/text()').extract()                   # 区域
            unitPrice = div.xpath('./div[6]/div[2]/span/text()').extract_first()    # 每平方米价格
            followInfo = div.xpath('./div[4]/text()').extract_first()               # 关注人数、发布时间
            item['title'] = title if title else ''
            item['url'] = url if url else ''
            item['region'] = ''.join(region) if region else ''
            item['unitPrice'] = unitPrice if unitPrice else ''
            # item['address'] = address if address else ''
            item['followInfo'] = followInfo if followInfo else ''
            item['name'] = 'room'
            # yield item
            yield scrapy.Request(item['url'], callback=self.crawl_r, meta={"item": item})

    def crawl_r(self, response):
        """爬取二手房详细信息"""
        item = dict()
        data = response.meta["item"]
        item.update(data)
        total = response.xpath('//span[@class="total"]/text()').extract_first()     # 房价
        visitTime = response.xpath('//div[@class="visitTime"]/span[2]/text()').extract_first()      # 看房时间
        lis = response.xpath('//div[@class="base"]/div[2]/ul')
        type = lis.xpath('./li[1]/text()').extract_first()                  # 房屋户型
        floor = lis.xpath('./li[2]/text()').extract_first()                 # 所在楼层
        floor_area = lis.xpath('./li[3]/text()').extract_first()            # 建筑面积
        structure = lis.xpath('./li[4]/text()').extract_first()             # 户型结构
        inside_area = lis.xpath('./li[5]/text()').extract_first()           # 套内面积
        building_type = lis.xpath('./li[6]/text()').extract_first()         # 建筑类型
        orientation = lis.xpath('./li[7]/text()').extract_first()           # 房屋朝向
        building_structure = lis.xpath('./li[8]/text()').extract_first()    # 建筑结构
        renovation = lis.xpath('./li[9]/text()').extract_first()            # 装修情况
        ladder = lis.xpath('./li[10]/text()').extract_first()               # 梯户比例
        elevator = lis.xpath('./li[11]/text()').extract_first()             # 配备电梯
        item["total"] = total if total else ""
        item["visitTime"] = visitTime if visitTime else ""
        item["type"] = type if type else ""
        item["floor"] = floor if floor else ""
        item["floor_area"] = floor_area if floor_area else ""
        item["structure"] = structure if structure else ""
        item["inside_area"] = inside_area if inside_area else ""
        item["building_type"] = building_type if building_type else ""
        item["orientation"] = orientation if orientation else ""
        item["building_structure"] = building_structure if building_structure else ""
        item["renovation"] = renovation if renovation else ""
        item["ladder"] = ladder if ladder else ""
        item["elevator"] = elevator if elevator else ""
        yield item


